import gzip, MySQLdb, nltk, json, sys
from MySQLdb import cursors
from collections import Counter

punct = [',','.',':','!',"", "(", ")", "[", "]"]
mystopwords = set(nltk.corpus.stopwords.words('english')+punct)

def lemmas_used(text):
	wnl = nltk.WordNetLemmatizer()
	sentences = nltk.sent_tokenize(text)
	sentences = [nltk.word_tokenize(sent) for sent in sentences]
	result = Counter()
	for sentence in sentences:
		for token in sentence:
			word = token.lower().strip(".-!/;'")
			if not word in mystopwords:
				result[wnl.lemmatize(word)] += 1
	return result

BookIDs = []
line = "start"
with gzip.open("books.txt.gz", 'r') as orfile:
	while line != '':
		line = orfile.readline()
		BookIDs.append(line.strip())
		print len(BookIDs), " out of 929,264 read"

BookIDs.pop()
BookIDs = set(BookIDs)
print "Loaded BookId's"

conn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
print "Connect to database"
#desc_reader = conn.cursor(cursors.SSCursor)
dict_writer = conn.cursor()
print "Created cursors"

#---------------------------------------------------------------------
To_Add = []
#print "Executing SELECT statement"
#desc_reader.execute('''SELECT * FROM Descriptions LIMIT 0 , 10''')
print "Opening file: descriptions"
myfile = gzip.open("descriptions.txt.gz",'r')

count_items = 0
count_books = 0
pr_id = myfile.readline()
while pr_id != '':
	print "Got new description"
	count_items += 1
	descr = myfile.readline()
	emt_l = myfile.readline()
	productID = ":".join(pr_id.strip().split(":")[1:]).strip()
	description = ":".join(descr.strip().split(":")[1:]).strip()
	print "Checking if description refers to book"
	if productID in BookIDs:
		print "It's a book. Creating a dictionary"
		count_books += 1
		dictionary = lemmas_used(description)
		To_Add.append((productID,json.dumps(dictionary)))
	if len(To_Add)%500 == 0:
		print "Gathered lots of descriptions"
		dict_writer.executemany('''INSERT INTO NDescriptions VALUES (%s,%s)''', To_Add)
		conn.commit()
		print "Added to database"
		To_Add = []
	print "Items: ", count_items, " , Books: ", count_books
	pr_id = myfile.readline()

myfile.close()
dict_writer.executemany('''INSERT INTO NDescriptions VALUES (%s,%s)''', To_Add)
conn.commit()
conn.close()

sys.exit()


myfile = gzip.open("descriptions.txt.gz",'r')

cc = Counter()
def test():
	pr_id = "0"
	while pr_id != "":
		pr_id = myfile.readline()
		descr = myfile.readline()
		emt_l = myfile.readline()
		productID = ":".join(pr_id.strip().split(":")[1:]).strip()
		description = ":".join(descr.strip().split(":")[1:]).strip()
		cc[productID in BookIDs] += 1

myfile.close()




